package org.correct.parser;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.util.Deque;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.NoSuchElementException;

import org.correct.config.Config;
import org.correct.index.IndexParser;

import org.correct.model.PageInfoItem;
import org.correct.util.PageInfoAccessor;


public class Parser {
	
	private enum TagType {
		doctype, comment, script, style, otherStart, otherEnd, single
	}
	
	private class Tag {
		public TagType type;
		public String name;
		
		public Tag(TagType type, String name) {
			this.type = type;
			this.name = name;
		}
	}
	
	private class Layer {
		StringBuffer buf;
		String tag;
		int weight;
		
		public Layer(int weight, String tag) {
			buf = new StringBuffer();
			this.weight = weight;
			this.tag = tag;
		}
		
		public String getContent() {
			return buf.toString();
		}
		
		public void refreshContent() {
			buf = new StringBuffer();
		}
	}
	
	private HashMap<String, Boolean> singleTags;
	
	private String srcFileBase;
	private String textBase;
	private Reader r = null;
	private PrintWriter wText = null;
	private int currentChar = 0;
	private Deque<Layer> layerStack;
	private Layer currentLayer;
	
	private static final int SINGLE_QUOTE = 1;
	private static final int DOUBLE_QUOTE = 2;
	private static final int NO_QUOTE = 0;
	
	private int quoteState = Parser.NO_QUOTE; 
	
	private char[] cbuf = null;
	private int bp = 0;
	private int bsize = 0;
	private static final int CHAR_BUF_SIZE = 1024;
	
	private HashMap<String, Integer> tagWeight;
	private PageInfoAccessor pageAccessor;
	private HashMap<Integer, Object> titleMap;
	private int currentId;
	
	private IndexParser termTable;
	
	public static int docNum = 0;
	
	public Parser(String srcFileBase, String textBase) {
		this.srcFileBase = srcFileBase;
		this.textBase = textBase;
		
		File textDir = new File(textBase);
		if(!textDir.exists()) {
			textDir.mkdir();
		} else if(textDir.isDirectory() == false) {
			textDir.delete();
			textDir.mkdir();
		}
		
		initTagWeight();
		initLooseTags();
		pageAccessor = new PageInfoAccessor(Config.PAGE_INFO_PATH);
		pageAccessor.load();
		titleMap = new HashMap<Integer, Object>();
		cbuf = new char[Parser.CHAR_BUF_SIZE];
	}
	
	private void initTagWeight() {
		tagWeight = new HashMap<String, Integer>();
		try {
			BufferedReader r = new BufferedReader(new FileReader("tags.txt"));
			String s = null;
			String[] args = null;
			while((s = r.readLine()) != null) {
				args = s.split("=");
				if(args.length < 2)
					continue;
				tagWeight.put(args[0], Integer.parseInt(args[1]));
			}
			r.close();
			
			// System.out.println(tagWeight);
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	private void initLooseTags() {
		singleTags = new HashMap<String, Boolean>();
		singleTags.put("br", true);
		singleTags.put("embed", true);
		singleTags.put("hr", true);
		singleTags.put("img", true);
		singleTags.put("input", true);
		singleTags.put("link", true);
		singleTags.put("meta", true);
		singleTags.put("p", true);
	}
	
	public void parse() {
		System.out.println("Parsing...");
		
		File index = new File("data/index");
		if(!index.exists()){
			index.mkdirs();
		}else{
			File[] indexs = index.listFiles();
			for(int i = 0; i < indexs.length; i++){
				indexs[i].delete();
			}
		}
		File dic = new File("data/dic");
		if(!dic.exists()){
			dic.mkdirs();
		}else{
			File[] dics = dic.listFiles();
			for(int i = 0; i < dics.length; i++){
				dics[i].delete();
			}
		}
		IndexParser.getStopWords();
		File srcDir = new File(srcFileBase);
		File[] srcList = srcDir.listFiles();
		docNum = srcList.length;
		for(int i = 0; i < srcList.length; i++){
			try{
				String[] s = srcList[i].getName().split("\\.");
				termTable = new IndexParser(Integer.parseInt(s[0]));
			}catch (Exception e) {
				e.printStackTrace();
				System.out.println("Read " + srcList[i].getName() + " error.");
			}
			
			System.out.println("Processing file " + srcList[i].getName() + " ...");
			
			currentId = getId(srcList[i].getName());
			PageInfoItem pageItem = pageAccessor.getItem(currentId);
			if(pageItem == null)	// page id out of range
				continue;
			
			parse(srcList[i]);
			termTable.write();
		}
		System.out.println("Parsing finished...");
		pageAccessor.updateInfo(PageInfoItem.InfoType.title, titleMap);
		pageAccessor.savePageInfo();
	}
	
	private int getId(String fileName) {
		int iDot = fileName.indexOf('.');
		if(iDot >= 0)
			return Integer.parseInt(fileName.substring(0, iDot));
		else
			return Integer.parseInt(fileName);
	}
	
	private void testParse(File f, String encoding) {
		File index = new File("data/index");
		if(!index.exists()){
			index.mkdirs();
		}else{
			File[] indexs = index.listFiles();
			for(int i = 0; i < indexs.length; i++){
				indexs[i].delete();
			}
		}
		File dic = new File("data/dic");
		if(!dic.exists()){
			dic.mkdirs();
		}else{
			File[] dics = dic.listFiles();
			for(int i = 0; i < dics.length; i++){
				dics[i].delete();
			}
		}
		IndexParser.getStopWords();
		
		termTable = new IndexParser(999);
		
		try {
			r = new InputStreamReader(new FileInputStream(f), encoding);
			initBuf();
			wText = new PrintWriter(f.getName() + ".txt", encoding);
			
			layerStack = new LinkedList<Layer>();
			currentChar = 0;
			
			readRootLayer();
			
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	private void parse(File f) {
		currentId = getId(f.getName());
		PageInfoItem pageItem = pageAccessor.getItem(currentId);
		
		String encoding = pageItem.encoding;
		
		try {
			r = new InputStreamReader(new FileInputStream(f), encoding);
			initBuf();
			wText = new PrintWriter(textBase + "/" + currentId + ".txt", encoding);
			
			layerStack = new LinkedList<Layer>();
			currentChar = 0;
			
			readRootLayer();
			
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	private void readRootLayer() {
		while(true) {
			// skip all characters before the first tag
			gotoLt();
			
			if(isFileEnd()) {
				clean();
				return;
			}
			
			Tag tag = readTag();
			// System.out.println("Root Layer: " + tag.name + "(" + tag.type + ")");
			
			if(tag.type == TagType.comment)
				skipComment();
			else if(tag.type == TagType.script)
				skipScript();
			else if(tag.type == TagType.style)
				skipStyle();
			else if(tag.type == TagType.doctype) {
				// just skip the tag
			} else if(tag.type == TagType.otherStart){
				
				if(currentLayer != null) {
					String content = currentLayer.getContent();
					
					wText.print(content + (content.length() > 0 ? " " : ""));
					termTable.parse(content, currentLayer.weight);
					currentLayer.refreshContent();
				}
				
				Integer w = tagWeight.get(tag.name);
				if(w == null)
					currentLayer = new Layer(1, tag.name);
				else
					currentLayer = new Layer(w, tag.name);
				layerStack.push(currentLayer);
				readLayer();
			} else if(tag.type == TagType.single) {
				// just skip single tags
			} else {
				
				if(currentLayer == null) {
					clean();
					return;
				}
					
				
				if(!tag.name.equalsIgnoreCase(currentLayer.tag))
					continue;
				// meet the closing tag
				String content = currentLayer.getContent();
				
				if(tag.name.equalsIgnoreCase("title"))
					titleMap.put(currentId, content);
				
				// System.out.println(content);
				
				wText.print(content + (content.length() > 0 ? " " : ""));
				termTable.parse(content, currentLayer.weight);
				break;
			}
		}
		
		try {
			clean();
			r.close();
			wText.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	private void readLayer() {
		while(true) {
			// write spaces and text
			while(true) {
				skipSpaces();
				
				if(currentChar == -1) {
					clean();
					return;
				} else if(currentChar == '<'){
					break;
				}
				
				readChars();
				
				if(currentChar == -1) {
					clean();
					return;
				} else if(currentChar == '<'){
					break;
				}
			}
			
			Tag tag = readTag();
			
			// System.out.println(currentId + ": " + tag.name + "(" + tag.type + ")");
			
			if(tag.type == TagType.comment)
				skipComment();
			else if(tag.type == TagType.script)
				skipScript();
			else if(tag.type == TagType.style)
				skipStyle();
			else if(tag.type == TagType.doctype) {
				// do nothing, just skip the tag
			} else if(tag.type == TagType.otherStart){
				
				if(currentLayer != null) {
					String content = currentLayer.getContent();
					
					wText.print(content + (content.length() > 0 ? " " : ""));
					termTable.parse(content, currentLayer.weight);
					currentLayer.refreshContent();
				}
				
				Integer w = tagWeight.get(tag.name);
				if(w == null)
					currentLayer = new Layer(currentLayer.weight, tag.name);
				else
					currentLayer = new Layer(currentLayer.weight + w, tag.name);
				layerStack.push(currentLayer);
				readLayer();
			} else if(tag.type == TagType.single) {
				// do nothing, just skip the tag
			} else {
				// meet the closing tag
				
				if(currentLayer == null) {
					clean();
					return;
				}
				
				if(!tag.name.equalsIgnoreCase(currentLayer.tag))
					continue;
				
				String content = currentLayer.getContent();
				
				if(tag.name.equalsIgnoreCase("title")) {
					// System.out.println("Title!!!! " + currentId + ": " + content);
					titleMap.put(currentId, content);
				}
				
				// System.out.println(content);
				
				wText.print(content + (content.length() > 0 ? " " : ""));
				termTable.parse(content, currentLayer.weight);
				try {
					layerStack.pop();
					currentLayer = layerStack.peek();
				} catch (NoSuchElementException e) {
				}
				return;
			}
		}
	}
	
	private void initBuf() {
		bp = -1;
		bsize = 0;
	}

	private int nextChar() throws IOException {
		bp++;
		if(bp >= bsize) {
			bsize = r.read(cbuf);
			if(bsize == -1)
				return -1;
			bp = 0;
		}
		
		return cbuf[bp];
	}
	
	private void skipChars(int numOfCharsToSkip) throws IOException {
		int toSkip = bp + numOfCharsToSkip;
		
		while(toSkip >= bsize) {
			toSkip = toSkip - bsize;
			bsize = r.read(cbuf);
			if(bsize == -1) {
				bp = -1;
				return;
			}
		}
		bp = toSkip;
	}
	
	private void readChars() {
		try {
			while(currentChar != -1 && !Character.isWhitespace(currentChar) &&
					currentChar != '<') {
				currentLayer.buf.append((char)currentChar);
				// System.out.println((char)currentChar);
				currentChar = nextChar();
			}
			currentLayer.buf.append(' ');
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	private void skipStyle() {
		try {
			while(true) {
				while(currentChar != -1 && currentChar != '<') {
					currentChar = nextChar();
				}
				if(currentChar == -1)
					return;
				if(currentChar == '<') {
					currentChar = nextChar();
					if(currentChar == '/') {
						currentChar = nextChar();
						if(currentChar == 's' || currentChar == 'S') {
							currentChar = nextChar();
							if(currentChar == 't' || currentChar == 'T') {
								currentChar = nextChar();
								if(currentChar == 'y' || currentChar == 'Y') {
									// nextChar();	// L
									// nextChar();	// E
									// nextChar();	// >
									// currentChar = nextChar();
									skipChars(3);	// LE>
									currentChar = nextChar();
									return;
								}
							}
						}
					}
				}
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
		
	}

	private void skipScript() {
		try {
			while(true) {
				while(currentChar != -1 && currentChar != '<') {
					currentChar = nextChar();
				}
				if(currentChar == -1)
					return;
				if(currentChar == '<') {
					currentChar = nextChar();
					if(currentChar == '/') {
						currentChar = nextChar();
						if(currentChar == 's' || currentChar == 'S') {
							currentChar = nextChar();
							if(currentChar == 'c' || currentChar == 'C') {
								// nextChar();	// R
								// nextChar();	// I
								// nextChar();	// P
								// nextChar();	// T
								// nextChar();	// >
								skipChars(5);	// RIPT>
								currentChar = nextChar();
								return;
							}
						}
					}
				}
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	private void skipTag() {
		try {
			while(currentChar != -1 && currentChar != '>')
				currentChar = nextChar();
			if(currentChar == '>')
				currentChar = nextChar();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	private void skipComment() {
		int barCount = 0;
		try {
			while(true) {
				while(currentChar != -1 && currentChar != '-') {
					currentChar = nextChar();
				}
				if(currentChar == -1)
					return;
				barCount = 0;
				while(currentChar == '-') {
					currentChar = nextChar();
					barCount++;
				}
				
				if(currentChar == '>' && barCount >= 2)
					return;
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	private void clean() {
		while(currentLayer != null) {
			String content = currentLayer.getContent();
			wText.print(content + (content.length() > 0 ? " " : ""));
			termTable.parse(content, currentLayer.weight);
			try {
				currentLayer = layerStack.pop();
			} catch (NoSuchElementException e) {
				break;
			}
		}
		
		try {
			wText.flush();
			r.close();
			wText.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	private boolean isFileEnd() {
		return (currentChar == -1);
	}
	
	private Tag readTag() {
		StringBuffer buf = new StringBuffer();
		boolean isEnd = false;
		boolean isSingle = false;
		String tagName = null;
		
		try {
			currentChar = nextChar();
			if(currentChar == '!') {
				buf.append((char)currentChar);
				currentChar = nextChar();
				buf.append((char)currentChar);
				if(currentChar == '-') {
					nextChar();	// -
					currentChar = nextChar();
					return new Tag(TagType.comment, null);
				} else if(currentChar == 'D' || currentChar == 'd') {
					/*
					nextChar();	// O
					nextChar();	// C
					nextChar();	// T
					nextChar();	// Y
					nextChar();	// P
					nextChar();	// E
					*/
					skipChars(6);	// OCTYPE
					currentChar = nextChar();
					skipTag();
					return new Tag(TagType.doctype, null);
				}
			} else if(currentChar == 's' || currentChar == 'S') {
				buf.append((char)currentChar);
				currentChar = nextChar();
				buf.append((char)currentChar);
				if(currentChar == 'c' || currentChar == 'C') {
					currentChar = nextChar();
					buf.append((char)currentChar);
					if(currentChar == 'r' || currentChar == 'R') {
						/*
						nextChar();	// I
						nextChar();	// P
						nextChar();	// T
						*/
						skipChars(3);	// IPT
						currentChar = nextChar();
						skipTag();
						return new Tag(TagType.script, null);
					}
				} else if(currentChar == 't' || currentChar == 'T') {
					currentChar = nextChar();
					buf.append((char)currentChar);
					if(currentChar == 'y' || currentChar == 'Y') {
						/*
						nextChar();	// L
						nextChar();	// E
						*/
						skipChars(2);
						currentChar = nextChar();
						skipTag();
						return new Tag(TagType.style, null);
					}
				}
			} else if(currentChar == '/') {
				isEnd = true;
			} else {
				buf.append((char)currentChar);
			}
			do {
				currentChar = nextChar();
				buf.append((char)currentChar);
			} while(currentChar != -1 && !Character.isWhitespace(currentChar)
					&& currentChar != '/' && currentChar != '>');
			
			buf.deleteCharAt(buf.length() - 1);
			tagName = buf.toString();
			
			gotoTagEnd();
			
			if(currentChar == '/') {
				isSingle = true;
				nextChar();	// >
			}
			if(currentChar != -1)
				currentChar = nextChar();
			if(singleTags.containsKey(tagName))
				isSingle = true;
			
		} catch (IOException e) {
			e.printStackTrace();
		}
		
		Tag tag = null;
		
		if(isSingle) {
			tag = new Tag(TagType.single, tagName);
		} else if(isEnd) {
			tag = new Tag(TagType.otherEnd, tagName);
		} else {
			tag = new Tag(TagType.otherStart, tagName);
		}
		
		return tag;
	}
	
	private void gotoTagEnd() {
		while(currentChar != '>' && currentChar != '/' && currentChar != -1) {
			/*
			if(!traceQuoteInTag())
				break;
			skipQuote();
			*/
			
			if(!traceProperty())
				break;
			skipProperty();
		}
	}
	
	private boolean traceProperty() {
		try {
			while(currentChar != '=' && currentChar != '>' && 
					 currentChar != '/' && currentChar != -1)
				currentChar = nextChar();
			if(currentChar == '=') {
				currentChar = nextChar();
				return true;
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
		return false;
	}
	
	private void skipProperty() {
		try {
			if(currentChar == '"') {
				quoteState = Parser.DOUBLE_QUOTE;
				currentChar = nextChar();
				skipQuote();
				return;
			} else if(currentChar == '\'') {
				quoteState = Parser.SINGLE_QUOTE;
				currentChar = nextChar();
				skipQuote();
				return;
			}
			
			while(!Character.isWhitespace(currentChar) && currentChar != '>'
				&& currentChar != -1)
				currentChar = nextChar();
			if(Character.isWhitespace(currentChar))
				currentChar = nextChar();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	private boolean traceQuoteInTag() {
		try {
			while(currentChar != '"' && currentChar != '>' && 
					 currentChar != '/' && currentChar != '\'' && currentChar != -1)
				currentChar = nextChar();
			if(currentChar == '"') {
				quoteState = Parser.DOUBLE_QUOTE;
				currentChar = nextChar();
				return true;
			} else if(currentChar == '\'') {
				quoteState = Parser.SINGLE_QUOTE;
				currentChar = nextChar();
				return true;
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
		return false;
	}
	
	private void skipQuote() {
		try {
			if(quoteState == Parser.SINGLE_QUOTE) {
				while(currentChar != '\'' && currentChar != -1)
					currentChar = nextChar();
				if(currentChar == '\'')
					currentChar = nextChar();
				quoteState = Parser.NO_QUOTE;
			} else if(quoteState == Parser.DOUBLE_QUOTE) {
				while(currentChar != '"' && currentChar != -1)
					currentChar = nextChar();
				if(currentChar == '"')
					currentChar = nextChar();
				quoteState = Parser.NO_QUOTE;
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	private void skipSpaces() {
		try {
			while(Character.isWhitespace(currentChar) && currentChar != -1) {
				currentChar = nextChar();
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	private void gotoLt() {
		if(currentChar == -1 || currentChar == '<')
			return;
		
		try {
			do {
				currentChar = nextChar();
			} while(currentChar != '<' && currentChar != -1);
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	public static void main(String args[]) {
		Parser p = new Parser("html", "text");
		// p.parse();
		p.testParse(new File("test.txt"), "utf-8");
	}
}
